# -*- coding: utf-8 -*-

# Scrapy settings for hk_bigdata project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'hk_bigdata'

SPIDER_MODULES = ['hk_bigdata.spiders']
NEWSPIDER_MODULE = 'hk_bigdata.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'hk_bigdata (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
COOKIES_ENABLED = False


# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    "Content-Type": "application/json",
    "Cookie": "pgv_pvi=449881088;RK=1FQxt9yWba;ptcz=a670c4a411ee07bcef8ccaf5bd204e7866325f3a14fbac58c9ff4b09ddca4d22;tvfe_boss_uuid=eda0ab25915e56c3;iip=0;eas_sid=p1S5Y9O8Q5H1s243u4G9V7F794;LOLWebSet_AreaBindInfo_1639873272=%257B%2522areaid%2522%253A%25221%2522%252C%2522areaname%2522%253A%2522%25E8%2589%25BE%25E6%25AC%25A7%25E5%25B0%25BC%25E4%25BA%259A%2520%25E7%2594%25B5%25E4%25BF%25A1%2522%252C%2522sRoleId%2522%253A0%252C%2522roleid%2522%253A%25221639873272%2522%252C%2522rolename%2522%253A%2522%25E4%25B8%25B6%25E6%2595%258C%25E6%2596%25B9%25E4%25B8%25A8%25E9%2598%25B2%25E5%25BE%25A1%25E5%25A1%2594%25E5%25A1%2594%2522%252C%2522checkparam%2522%253A%2522lol%257Cyes%257C1639873272%257C1%257C1639873272*%257C%257C%257C%257C%2525E4%2525B8%2525B6%2525E6%252595%25258C%2525E6%252596%2525B9%2525E4%2525B8%2525A8%2525E9%252598%2525B2%2525E5%2525BE%2525A1%2525E5%2525A1%252594%2525E5%2525A1%252594*%257C%257C%257C1602308175%257C%2522%252C%2522md5str%2522%253A%252294E94A518C61EB349017405193F95901%2522%252C%2522roleareaid%2522%253A%25221%2522%252C%2522sPartition%2522%253A%25221%2522%257D;LOLWebSet_AreaBindInfo_2564129056=%257B%2522areaid%2522%253A%25221%2522%252C%2522areaname%2522%253A%2522%25E8%2589%25BE%25E6%25AC%25A7%25E5%25B0%25BC%25E4%25BA%259A%2520%25E7%2594%25B5%25E4%25BF%25A1%2522%252C%2522sRoleId%2522%253A0%252C%2522roleid%2522%253A%25222564129056%2522%252C%2522rolename%2522%253A%2522%25E8%258C%2597%25E5%25A4%25A9%25E4%25BE%259D%2522%252C%2522checkparam%2522%253A%2522lol%257Cyes%257C2564129056%257C1%257C2564129056*%257C%257C%257C%257C%2525E8%25258C%252597%2525E5%2525A4%2525A9%2525E4%2525BE%25259D*%257C%257C%257C1602308197%257C%2522%252C%2522md5str%2522%253A%2522395B392DBB8D193AD238F687E05D4D40%2522%252C%2522roleareaid%2522%253A%25221%2522%252C%2522sPartition%2522%253A%25221%2522%257D;pvid=4641524768;pac_uid=1_643080304;pgv_pvid=7127563704;o_cookie=643080304;uin_cookie=o0643080304;ied_qq=o0643080304;LOLWebSet_AreaBindInfo_643080304=%257B%2522areaid%2522%253A%25221%2522%252C%2522areaname%2522%253A%2522%25E8%2589%25BE%25E6%25AC%25A7%25E5%25B0%25BC%25E4%25BA%259A%2520%25E7%2594%25B5%25E4%25BF%25A1%2522%252C%2522sRoleId%2522%253A0%252C%2522roleid%2522%253A%2522643080304%2522%252C%2522rolename%2522%253A%2522%25E5%25B0%258F%25E8%258E%25B9%25E8%25BD%25BB%25E7%2582%25B9%25E7%2596%25BC%2522%252C%2522checkparam%2522%253A%2522lol%257Cyes%257C643080304%257C1%257C643080304*%257C%257C%257C%257C%2525E5%2525B0%25258F%2525E8%25258E%2525B9%2525E8%2525BD%2525BB%2525E7%252582%2525B9%2525E7%252596%2525BC*%257C%257C%257C1611203418%257C%2522%252C%2522md5str%2522%253A%2522413CC1D2A4B6A597776B2A0ED0621126%2522%252C%2522roleareaid%2522%253A%25221%2522%252C%2522sPartition%2522%253A%25221%2522%257D;LW_uid=h166g1e1J2g033e4x631L1j4i0;LW_sid=a1x6x1B1Q2B00384l9Y9Q2y7q3;sd_userid=40141612143891252;sd_cookie_crttime=1612143891252;heat_sess_police=MTYyMDk2MDkwNnx0WDFZcFBRSHdNNXVYY0pLQ2JHdmdQR3pfZXd6OGc5WXpUdmFFdGFyZGJwcDBWNE15U2c1dHpnRHZyWkRpMGp4U0YxX0N6WTEwdjVsZ2RDUVdDQ0lKZ19ZX3pJbEZqeGhqb0YyTUNtZTNTYzRfWWdfU2o5NE1zNER3VUhfZkJrLVhDVFRtOGpZNjJDby0xc3pybkxZckREMUNRMTlMTzFObUVxdXloS05SOGh4cU9PS0hQdmpYTjllVlc3c0tfVUxkblNQcXowSkZNTUNQejJmUGFDT05aMDB5aFRFUnBfaXZSUmZPZEhlSjU2Rjh3QjFxanBXMVVaZHNhOFpZREpjUFl0cHNfdmNMazhnUlJTbFVuOG53QzZDWDB3b1NGbzAzcGVqcWs1U1c2dEh1QWNyNTAxMFU1czdxMnFxbEV3a3cxQnpxRjI3VkhUVmo0ZEswbWhyWkR3bW9ILUtIaXlGbWR2aVlPM1lXdl9INW52NktCU1lCd2lDbWhvR0YyYXp6YVhNc3pFbTNkWGhWUmhRY1lYRXlZTDZ0bWhHU2ZHWGh1b0xaQXBWZmlsdV9nMEczUWR1cG1UWDNIZEtfOEpTSVliWUNValIxTWkwMHo5SWZyRnZHNGFRTFM2Z0NYcUp6clFQQXdhZ0RBNi1fQVBHVU9KZ196R0dCU1pQYVA2TXRjUTJzVFduT0lTdnFTRmktRGRMYUp0OGFpTVEzTlJKd240ZmhNM3NVajJmRk1rb2VUX2FVWXh3ZV9BbV9ReUNXbVlvQ0hnYjgyeHhva1NBWlM2MjdYRURkYkNYaFZnT05wS3Z4OEZiZDBjelMyVHhIdjk5VUdBOHowLXA4WGxpVDY1ZTZRPT18yxB-WOulqNmMP6zby5N5ecxqHqKx0FsA-zVasE3DI5w=",
    "User-Agent": "Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/90.0.4430.212Safari/537.36"
}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'hk_bigdata.middlewares.HkBigdataSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'hk_bigdata.middlewares.HkBigdataDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'hk_bigdata.pipelines.MysqlPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'


MYSQL_HOST = 'localhost'
MYSQL_DATABASE = 'hk_bigdata'
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'root'
MYSQL_PORT = 3306